{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "editable": false,
    "id": "-r-SJUrp6b_t"
   },
   "source": [
    "## Setting up"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 注意⚠️\n",
    "如果huggingface 无法连接，deepseek-r1-llama-8b模型和数据集无法下载。因此为了方便大家，提供一个网盘下载地址，请到该地址，即可下载模型和相应的训练数据集：\n",
    "> 模型下载地址：https://pan.quark.cn/s/d45a88061e3d\n",
    "> 训练数据集下载地址：https://pan.quark.cn/s/fea6a395deff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing to /home/user/.config/pip/pip.conf\n"
     ]
    }
   ],
   "source": [
    "!pip config set global.index-url https://pypi.mirrors.ustc.edu.cn/simple # 修改pip下载源"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "editable": false,
    "id": "bmDhM2646b_v",
    "papermill": {
     "duration": 164.985343,
     "end_time": "2024-09-25T13:11:54.926296",
     "exception": false,
     "start_time": "2024-09-25T13:09:09.940953",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.mirrors.ustc.edu.cn/simple\n",
      "Requirement already satisfied: unsloth in /app/anaconda/weitiao/lib/python3.12/site-packages (2025.2.12)\n",
      "Looking in indexes: https://pypi.mirrors.ustc.edu.cn/simple\n",
      "Collecting git+https://github.com/unslothai/unsloth.git\n",
      "  Cloning https://github.com/unslothai/unsloth.git to /tmp/pip-req-build-kh25m9nd\n",
      "  Running command git clone --filter=blob:none --quiet https://github.com/unslothai/unsloth.git /tmp/pip-req-build-kh25m9nd\n",
      "  fatal: unable to access 'https://github.com/unslothai/unsloth.git/': GnuTLS recv error (-110): The TLS connection was non-properly terminated.\n",
      "  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
      "  \n",
      "  \u001b[31m×\u001b[0m \u001b[32mgit clone --\u001b[0m\u001b[32mfilter\u001b[0m\u001b[32m=\u001b[0m\u001b[32mblob\u001b[0m\u001b[32m:none --quiet \u001b[0m\u001b[4;32mhttps://github.com/unslothai/unsloth.git\u001b[0m\u001b[32m \u001b[0m\u001b[32m/tmp/\u001b[0m\u001b[32mpip-req-build-kh25m9nd\u001b[0m did not run successfully.\n",
      "  \u001b[31m│\u001b[0m exit code: \u001b[1;36m128\u001b[0m\n",
      "  \u001b[31m╰─>\u001b[0m See above for output.\n",
      "  \n",
      "  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
      "\u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
      "\n",
      "\u001b[31m×\u001b[0m \u001b[32mgit clone --\u001b[0m\u001b[32mfilter\u001b[0m\u001b[32m=\u001b[0m\u001b[32mblob\u001b[0m\u001b[32m:none --quiet \u001b[0m\u001b[4;32mhttps://github.com/unslothai/unsloth.git\u001b[0m\u001b[32m \u001b[0m\u001b[32m/tmp/\u001b[0m\u001b[32mpip-req-build-kh25m9nd\u001b[0m did not run successfully.\n",
      "\u001b[31m│\u001b[0m exit code: \u001b[1;36m128\u001b[0m\n",
      "\u001b[31m╰─>\u001b[0m See above for output.\n",
      "\n",
      "\u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n"
     ]
    }
   ],
   "source": [
    "# %%capture\n",
    "!pip install unsloth\n",
    "# Also get the latest nightly Unsloth!\n",
    "!pip install --force-reinstall --no-cache-dir --no-deps git+https://github.com/unslothai/unsloth.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "editable": false,
    "id": "G1UpWpAQ6b_w",
    "outputId": "6cf7bf5c-9c83-4d8f-bfa5-ea3c3c45f492",
    "papermill": {
     "duration": 31.301473,
     "end_time": "2024-09-25T13:12:26.233487",
     "exception": false,
     "start_time": "2024-09-25T13:11:54.932014",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING[XFORMERS]: xFormers can't load C++/CUDA extensions. xFormers was built for:\n",
      "    PyTorch 2.6.0+cu124 with CUDA 1204 (you have 2.6.0+cu126)\n",
      "    Python  3.12.9 (you have 3.12.9)\n",
      "  Please reinstall xformers (see https://github.com/facebookresearch/xformers#installing-xformers)\n",
      "  Memory-efficient attention, SwiGLU, sparse and more won't be available.\n",
      "  Set XFORMERS_MORE_DETAILS=1 for more details\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "🦥 Unsloth Zoo will now patch everything to make training faster!\n"
     ]
    }
   ],
   "source": [
    "from unsloth import FastLanguageModel\n",
    "import torch\n",
    "max_seq_length = 2048\n",
    "dtype = None\n",
    "load_in_4bit = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QMIlEq9dGDG0"
   },
   "source": [
    "可视化训练过程，将数据报送到wandb中"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.mirrors.ustc.edu.cn/simple\n",
      "Requirement already satisfied: wandb in /app/anaconda/weitiao/lib/python3.12/site-packages (0.19.6)\n",
      "Requirement already satisfied: click!=8.0.0,>=7.1 in /app/anaconda/weitiao/lib/python3.12/site-packages (from wandb) (8.1.8)\n",
      "Requirement already satisfied: docker-pycreds>=0.4.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from wandb) (0.4.0)\n",
      "Requirement already satisfied: gitpython!=3.1.29,>=1.0.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from wandb) (3.1.44)\n",
      "Requirement already satisfied: platformdirs in /app/anaconda/weitiao/lib/python3.12/site-packages (from wandb) (4.3.6)\n",
      "Requirement already satisfied: protobuf!=4.21.0,!=5.28.0,<6,>=3.19.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from wandb) (3.20.3)\n",
      "Requirement already satisfied: psutil>=5.0.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from wandb) (7.0.0)\n",
      "Requirement already satisfied: pydantic<3,>=2.6 in /app/anaconda/weitiao/lib/python3.12/site-packages (from wandb) (2.10.6)\n",
      "Requirement already satisfied: pyyaml in /app/anaconda/weitiao/lib/python3.12/site-packages (from wandb) (6.0.2)\n",
      "Requirement already satisfied: requests<3,>=2.0.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from wandb) (2.32.3)\n",
      "Requirement already satisfied: sentry-sdk>=2.0.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from wandb) (2.21.0)\n",
      "Requirement already satisfied: setproctitle in /app/anaconda/weitiao/lib/python3.12/site-packages (from wandb) (1.3.4)\n",
      "Requirement already satisfied: setuptools in /app/anaconda/weitiao/lib/python3.12/site-packages (from wandb) (75.8.0)\n",
      "Requirement already satisfied: six>=1.4.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from docker-pycreds>=0.4.0->wandb) (1.17.0)\n",
      "Requirement already satisfied: gitdb<5,>=4.0.1 in /app/anaconda/weitiao/lib/python3.12/site-packages (from gitpython!=3.1.29,>=1.0.0->wandb) (4.0.12)\n",
      "Requirement already satisfied: annotated-types>=0.6.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from pydantic<3,>=2.6->wandb) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.27.2 in /app/anaconda/weitiao/lib/python3.12/site-packages (from pydantic<3,>=2.6->wandb) (2.27.2)\n",
      "Requirement already satisfied: typing-extensions>=4.12.2 in /app/anaconda/weitiao/lib/python3.12/site-packages (from pydantic<3,>=2.6->wandb) (4.12.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /app/anaconda/weitiao/lib/python3.12/site-packages (from requests<3,>=2.0.0->wandb) (3.4.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /app/anaconda/weitiao/lib/python3.12/site-packages (from requests<3,>=2.0.0->wandb) (3.10)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /app/anaconda/weitiao/lib/python3.12/site-packages (from requests<3,>=2.0.0->wandb) (2.3.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /app/anaconda/weitiao/lib/python3.12/site-packages (from requests<3,>=2.0.0->wandb) (2025.1.31)\n",
      "Requirement already satisfied: smmap<6,>=3.0.1 in /app/anaconda/weitiao/lib/python3.12/site-packages (from gitdb<5,>=4.0.1->gitpython!=3.1.29,>=1.0.0->wandb) (5.0.2)\n"
     ]
    }
   ],
   "source": [
    "!pip install wandb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /home/user/.netrc\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mneuqkyn\u001b[0m (\u001b[33mneuqkyn-rhzz\u001b[0m) to \u001b[32mhttps://api.wandb.ai\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Using wandb-core as the SDK backend.  Please refer to https://wandb.me/wandb-core for more information.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.19.6"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/app/code/finetuning/wandb/run-20250219_145843-vrlu3wz2</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/neuqkyn-rhzz/my%20fint-tune%20on%20deepseek%20r1%20with%20medical%20data%202/runs/vrlu3wz2?apiKey=822b638c1f465585a2a139ce45f36c4d0a6e0180' target=\"_blank\">silvery-donkey-18</a></strong> to <a href='https://wandb.ai/neuqkyn-rhzz/my%20fint-tune%20on%20deepseek%20r1%20with%20medical%20data%202?apiKey=822b638c1f465585a2a139ce45f36c4d0a6e0180' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/developer-guide' target=\"_blank\">docs</a>)<br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/neuqkyn-rhzz/my%20fint-tune%20on%20deepseek%20r1%20with%20medical%20data%202?apiKey=822b638c1f465585a2a139ce45f36c4d0a6e0180' target=\"_blank\">https://wandb.ai/neuqkyn-rhzz/my%20fint-tune%20on%20deepseek%20r1%20with%20medical%20data%202?apiKey=822b638c1f465585a2a139ce45f36c4d0a6e0180</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/neuqkyn-rhzz/my%20fint-tune%20on%20deepseek%20r1%20with%20medical%20data%202/runs/vrlu3wz2?apiKey=822b638c1f465585a2a139ce45f36c4d0a6e0180' target=\"_blank\">https://wandb.ai/neuqkyn-rhzz/my%20fint-tune%20on%20deepseek%20r1%20with%20medical%20data%202/runs/vrlu3wz2?apiKey=822b638c1f465585a2a139ce45f36c4d0a6e0180</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Do NOT share these links with anyone. They can be used to claim your runs."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## 这里不使用环境变量，直接填入wandb的token,如果没有token可以去官网下载一个\n",
    "import wandb\n",
    "\n",
    "wandb.login(key=\"822b638c1f465585a2a139ce45f36c4d0a6e0180\")\n",
    "run = wandb.init(\n",
    "    project='my fint-tune on deepseek r1 with medical data 2',\n",
    "    job_type=\"training\",\n",
    "    anonymous=\"allow\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "editable": false,
    "id": "x3C_JY6e6b_x"
   },
   "source": [
    "## Loading the model and tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.mirrors.ustc.edu.cn/simple\n",
      "Requirement already satisfied: compressed-tensors in /app/anaconda/weitiao/lib/python3.12/site-packages (0.9.2)\n",
      "Requirement already satisfied: torch>=1.7.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from compressed-tensors) (2.6.0+cu126)\n",
      "Requirement already satisfied: transformers in /app/anaconda/weitiao/lib/python3.12/site-packages (from compressed-tensors) (4.49.0)\n",
      "Requirement already satisfied: pydantic>=2.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from compressed-tensors) (2.10.6)\n",
      "Requirement already satisfied: annotated-types>=0.6.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from pydantic>=2.0->compressed-tensors) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.27.2 in /app/anaconda/weitiao/lib/python3.12/site-packages (from pydantic>=2.0->compressed-tensors) (2.27.2)\n",
      "Requirement already satisfied: typing-extensions>=4.12.2 in /app/anaconda/weitiao/lib/python3.12/site-packages (from pydantic>=2.0->compressed-tensors) (4.12.2)\n",
      "Requirement already satisfied: filelock in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (3.13.1)\n",
      "Requirement already satisfied: setuptools in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (75.8.0)\n",
      "Requirement already satisfied: sympy==1.13.1 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (1.13.1)\n",
      "Requirement already satisfied: networkx in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (3.3)\n",
      "Requirement already satisfied: jinja2 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (3.1.4)\n",
      "Requirement already satisfied: fsspec in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (2024.6.1)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (12.6.77)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (12.6.77)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (12.6.80)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (9.5.1.17)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (12.6.4.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (11.3.0.4)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (10.3.7.77)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (11.7.1.2)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (12.5.4.2)\n",
      "Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (0.6.3)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.21.5 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (2.21.5)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (12.6.77)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (12.6.85)\n",
      "Requirement already satisfied: triton==3.2.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.7.0->compressed-tensors) (3.2.0)\n",
      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from sympy==1.13.1->torch>=1.7.0->compressed-tensors) (1.3.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.26.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from transformers->compressed-tensors) (0.28.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /app/anaconda/weitiao/lib/python3.12/site-packages (from transformers->compressed-tensors) (2.1.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from transformers->compressed-tensors) (24.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /app/anaconda/weitiao/lib/python3.12/site-packages (from transformers->compressed-tensors) (6.0.2)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /app/anaconda/weitiao/lib/python3.12/site-packages (from transformers->compressed-tensors) (2024.11.6)\n",
      "Requirement already satisfied: requests in /app/anaconda/weitiao/lib/python3.12/site-packages (from transformers->compressed-tensors) (2.32.3)\n",
      "Requirement already satisfied: tokenizers<0.22,>=0.21 in /app/anaconda/weitiao/lib/python3.12/site-packages (from transformers->compressed-tensors) (0.21.0)\n",
      "Requirement already satisfied: safetensors>=0.4.1 in /app/anaconda/weitiao/lib/python3.12/site-packages (from transformers->compressed-tensors) (0.5.2)\n",
      "Requirement already satisfied: tqdm>=4.27 in /app/anaconda/weitiao/lib/python3.12/site-packages (from transformers->compressed-tensors) (4.67.1)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from jinja2->torch>=1.7.0->compressed-tensors) (2.1.5)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /app/anaconda/weitiao/lib/python3.12/site-packages (from requests->transformers->compressed-tensors) (3.4.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /app/anaconda/weitiao/lib/python3.12/site-packages (from requests->transformers->compressed-tensors) (3.10)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /app/anaconda/weitiao/lib/python3.12/site-packages (from requests->transformers->compressed-tensors) (2.3.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /app/anaconda/weitiao/lib/python3.12/site-packages (from requests->transformers->compressed-tensors) (2025.1.31)\n"
     ]
    }
   ],
   "source": [
    "!pip install compressed-tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: compressed-tensors\n",
      "Version: 0.9.2\n",
      "Summary: Library for utilization of compressed safetensors of neural network models\n",
      "Home-page: https://github.com/neuralmagic/compressed-tensors\n",
      "Author: Neuralmagic, Inc.\n",
      "Author-email: support@neuralmagic.com\n",
      "License: Apache 2.0\n",
      "Location: /app/anaconda/weitiao/lib/python3.12/site-packages\n",
      "Requires: pydantic, torch, transformers\n",
      "Required-by: \n"
     ]
    }
   ],
   "source": [
    "!pip show compressed-tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==((====))==  Unsloth 2025.2.12: Fast Llama patching. Transformers: 4.49.0.\n",
      "   \\\\   /|    GPU: NVIDIA L20. Max memory: 44.422 GB. Platform: Linux.\n",
      "O^O/ \\_/ \\    Torch: 2.6.0+cu126. CUDA: 8.9. CUDA Toolkit: 12.6. Triton: 3.2.0\n",
      "\\        /    Bfloat16 = TRUE. FA [Xformers = None. FA2 = False]\n",
      " \"-____-\"     Free Apache license: http://github.com/unslothai/unsloth\n",
      "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b608f5701ec24f6dbcb84ac06d6cd55b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_file: None\n",
      "/app/models/DeepSeek-R1-Distill-Llama-8B does not have a padding token! Will use pad_token = <|finetune_right_pad_id|>.\n"
     ]
    }
   ],
   "source": [
    "model, tokenizer = FastLanguageModel.from_pretrained(\n",
    "    model_name = \"/app/models/DeepSeek-R1-Distill-Llama-8B\", # 这里改成你本地模型，以我的为例，我已经huggingface上的模型文件下载到本地。\n",
    "    max_seq_length = max_seq_length,\n",
    "    dtype = dtype,\n",
    "    load_in_4bit = False,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "editable": false,
    "id": "DHTrWVzp6b_x"
   },
   "source": [
    "## Model inference before fine-tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "editable": false,
    "id": "y36oW6mo6b_y",
    "papermill": {
     "duration": 0.014436,
     "end_time": "2024-09-25T13:13:09.483925",
     "exception": false,
     "start_time": "2024-09-25T13:13:09.469489",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "prompt_style = \"\"\"Below is an instruction that describes a task, paired with an input that provides further context.\n",
    "Write a response that appropriately completes the request.\n",
    "Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\n",
    "\n",
    "### Instruction:\n",
    "You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning.\n",
    "Please answer the following medical question.\n",
    "\n",
    "### Question:\n",
    "{}\n",
    "\n",
    "### Response:\n",
    "<think>{}\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataParallel(\n",
       "  (module): LlamaForCausalLM(\n",
       "    (model): LlamaModel(\n",
       "      (embed_tokens): Embedding(128256, 4096, padding_idx=128004)\n",
       "      (layers): ModuleList(\n",
       "        (0-31): 32 x LlamaDecoderLayer(\n",
       "          (self_attn): LlamaAttention(\n",
       "            (q_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "            (k_proj): Linear(in_features=4096, out_features=1024, bias=False)\n",
       "            (v_proj): Linear(in_features=4096, out_features=1024, bias=False)\n",
       "            (o_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "            (rotary_emb): LlamaRotaryEmbedding()\n",
       "          )\n",
       "          (mlp): LlamaMLP(\n",
       "            (gate_proj): Linear(in_features=4096, out_features=14336, bias=False)\n",
       "            (up_proj): Linear(in_features=4096, out_features=14336, bias=False)\n",
       "            (down_proj): Linear(in_features=14336, out_features=4096, bias=False)\n",
       "            (act_fn): SiLU()\n",
       "          )\n",
       "          (input_layernorm): LlamaRMSNorm((4096,), eps=1e-05)\n",
       "          (post_attention_layernorm): LlamaRMSNorm((4096,), eps=1e-05)\n",
       "        )\n",
       "      )\n",
       "      (norm): LlamaRMSNorm((4096,), eps=1e-05)\n",
       "      (rotary_emb): LlamaRotaryEmbedding()\n",
       "    )\n",
       "    (lm_head): Linear(in_features=4096, out_features=128256, bias=False)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将模型移动到多个 GPU 上\n",
    "device_ids = [0]  # 指定使用 cuda:0 和 cuda:1\n",
    "model = torch.nn.DataParallel(model, device_ids=device_ids)\n",
    "model.to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "<think>\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "question = \"A 59-year-old man presents with a fever, chills, night sweats, and generalized fatigue, and is found to have a 12 mm vegetation on the aortic valve. Blood cultures indicate gram-positive, catalase-negative, gamma-hemolytic cocci in chains that do not grow in a 6.5% NaCl medium. What is the most likely predisposing factor for this patient's condition?\"\n",
    "\n",
    "\n",
    "inputs = tokenizer([prompt_style.format(question, \"\")], return_tensors=\"pt\").to(\"cuda\")\n",
    "FastLanguageModel.for_inference(model.module)\n",
    "outputs = model.module.generate(\n",
    "    input_ids=inputs.input_ids,\n",
    "    attention_mask=inputs.attention_mask,\n",
    "    max_new_tokens=1200,\n",
    "    use_cache=True,\n",
    ")\n",
    "response = tokenizer.batch_decode(outputs)\n",
    "print(response[0].split(\"### Response:\")[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LlamaForCausalLM(\n",
       "  (model): LlamaModel(\n",
       "    (embed_tokens): Embedding(128256, 4096, padding_idx=128004)\n",
       "    (layers): ModuleList(\n",
       "      (0-31): 32 x LlamaDecoderLayer(\n",
       "        (self_attn): LlamaAttention(\n",
       "          (q_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "          (k_proj): Linear(in_features=4096, out_features=1024, bias=False)\n",
       "          (v_proj): Linear(in_features=4096, out_features=1024, bias=False)\n",
       "          (o_proj): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "          (rotary_emb): LlamaRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): LlamaMLP(\n",
       "          (gate_proj): Linear(in_features=4096, out_features=14336, bias=False)\n",
       "          (up_proj): Linear(in_features=4096, out_features=14336, bias=False)\n",
       "          (down_proj): Linear(in_features=14336, out_features=4096, bias=False)\n",
       "          (act_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): LlamaRMSNorm((4096,), eps=1e-05)\n",
       "        (post_attention_layernorm): LlamaRMSNorm((4096,), eps=1e-05)\n",
       "      )\n",
       "    )\n",
       "    (norm): LlamaRMSNorm((4096,), eps=1e-05)\n",
       "    (rotary_emb): LlamaRotaryEmbedding()\n",
       "  )\n",
       "  (lm_head): Linear(in_features=4096, out_features=128256, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FastLanguageModel.for_training(model.module)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "editable": false,
    "id": "PLzmU3OH6b_y",
    "outputId": "57e968e5-f7d8-4140-8fa1-05cf5409f219",
    "papermill": {
     "duration": 5.943269,
     "end_time": "2024-09-25T13:13:09.461941",
     "exception": false,
     "start_time": "2024-09-25T13:13:03.518672",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unsloth 2025.2.12 patched 32 layers with 32 QKV layers, 32 O layers and 32 MLP layers.\n"
     ]
    }
   ],
   "source": [
    "model = FastLanguageModel.get_peft_model(\n",
    "    model.module,\n",
    "    r=16,\n",
    "    target_modules=[\n",
    "        \"q_proj\",\n",
    "        \"k_proj\",\n",
    "        \"v_proj\",\n",
    "        \"o_proj\",\n",
    "        \"gate_proj\",\n",
    "        \"up_proj\",\n",
    "        \"down_proj\",\n",
    "    ],\n",
    "    lora_alpha=16,\n",
    "    lora_dropout=0,\n",
    "    bias=\"none\",\n",
    "    use_gradient_checkpointing=\"unsloth\",  # True or \"unsloth\" for very long context\n",
    "    random_state=3407,\n",
    "    use_rslora=False,\n",
    "    loftq_config=None,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "editable": false,
    "id": "x_trNlci6b_y"
   },
   "source": [
    "## Loading and processing the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "editable": false,
    "id": "CwCzOgAy6b_y"
   },
   "outputs": [],
   "source": [
    "train_prompt_style = \"\"\"Below is an instruction that describes a task, paired with an input that provides further context.\n",
    "Write a response that appropriately completes the request.\n",
    "Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\n",
    "\n",
    "### Instruction:\n",
    "You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning.\n",
    "Please answer the following medical question.\n",
    "\n",
    "### Question:\n",
    "{}\n",
    "\n",
    "### Response:\n",
    "<think>\n",
    "{}\n",
    "</think>\n",
    "{}\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "editable": false,
    "id": "Fo0OGx8w6b_z",
    "papermill": {
     "duration": 0.015203,
     "end_time": "2024-09-25T13:13:09.506291",
     "exception": false,
     "start_time": "2024-09-25T13:13:09.491088",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "EOS_TOKEN = tokenizer.eos_token  # Must add EOS_TOKEN\n",
    "\n",
    "\n",
    "def formatting_prompts_func(examples):\n",
    "    inputs = examples[\"Question\"]\n",
    "    cots = examples[\"Complex_CoT\"]\n",
    "    outputs = examples[\"Response\"]\n",
    "    texts = []\n",
    "    for input, cot, output in zip(inputs, cots, outputs):\n",
    "        text = train_prompt_style.format(input, cot, output) + EOS_TOKEN\n",
    "        texts.append(text)\n",
    "    return {\n",
    "        \"text\": texts,\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "id": "kSCrcJxlAfoa",
    "outputId": "9d6ddebb-e390-4035-f3aa-977486080305"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Below is an instruction that describes a task, paired with an input that provides further context.\\nWrite a response that appropriately completes the request.\\nBefore answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\\n\\n### Instruction:\\nYou are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning.\\nPlease answer the following medical question.\\n\\n### Question:\\nA 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?\\n\\n### Response:\\n<think>\\nOkay, let's think about this step by step. There's a 61-year-old woman here who's been dealing with involuntary urine leakages whenever she's doing something that ups her abdominal pressure like coughing or sneezing. This sounds a lot like stress urinary incontinence to me. Now, it's interesting that she doesn't have any issues at night; she isn't experiencing leakage while sleeping. This likely means her bladder's ability to hold urine is fine when she isn't under physical stress. Hmm, that's a clue that we're dealing with something related to pressure rather than a bladder muscle problem. \\n\\nThe fact that she underwent a Q-tip test is intriguing too. This test is usually done to assess urethral mobility. In stress incontinence, a Q-tip might move significantly, showing urethral hypermobility. This kind of movement often means there's a weakness in the support structures that should help keep the urethra closed during increases in abdominal pressure. So, that's aligning well with stress incontinence.\\n\\nNow, let's think about what would happen during cystometry. Since stress incontinence isn't usually about sudden bladder contractions, I wouldn't expect to see involuntary detrusor contractions during this test. Her bladder isn't spasming or anything; it's more about the support structure failing under stress. Plus, she likely empties her bladder completely because stress incontinence doesn't typically involve incomplete emptying. So, her residual volume should be pretty normal. \\n\\nAll in all, it seems like if they do a cystometry on her, it will likely show a normal residual volume and no involuntary contractions. Yup, I think that makes sense given her symptoms and the typical presentations of stress urinary incontinence.\\n</think>\\nCystometry in this case of stress urinary incontinence would most likely reveal a normal post-void residual volume, as stress incontinence typically does not involve issues with bladder emptying. Additionally, since stress urinary incontinence is primarily related to physical exertion and not an overactive bladder, you would not expect to see any involuntary detrusor contractions during the test.<｜end▁of▁sentence｜>\""
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "dataset = load_dataset(\"/app/datasets/medical-o1-reasoning-SFT\", \"en\",split = \"train[0:500]\") # 这里同样去huggingface上面下载数据集，然后放到本地\n",
    "dataset = dataset.map(formatting_prompts_func, batched = True,)\n",
    "dataset[\"text\"][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "editable": false,
    "id": "uFmyf5_w6b_z"
   },
   "source": [
    "## Setting up the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found existing installation: peft 0.14.0\n",
      "Uninstalling peft-0.14.0:\n",
      "  Successfully uninstalled peft-0.14.0\n",
      "Found existing installation: transformers 4.49.0\n",
      "Uninstalling transformers-4.49.0:\n",
      "  Successfully uninstalled transformers-4.49.0\n",
      "Found existing installation: trl 0.15.0\n",
      "Uninstalling trl-0.15.0:\n",
      "  Successfully uninstalled trl-0.15.0\n",
      "Found existing installation: accelerate 1.4.0\n",
      "Uninstalling accelerate-1.4.0:\n",
      "  Successfully uninstalled accelerate-1.4.0\n",
      "Found existing installation: bitsandbytes 0.45.2\n",
      "Uninstalling bitsandbytes-0.45.2:\n",
      "  Successfully uninstalled bitsandbytes-0.45.2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.mirrors.ustc.edu.cn/simple\n",
      "Collecting peft\n",
      "  Using cached https://mirrors.ustc.edu.cn/pypi/packages/88/05/e58e3aaa36544d30a917814e336fc65a746f708e5874945e92999bc22fa3/peft-0.14.0-py3-none-any.whl (374 kB)\n",
      "Collecting transformers\n",
      "  Using cached https://mirrors.ustc.edu.cn/pypi/packages/20/37/1f29af63e9c30156a3ed6ebc2754077016577c094f31de7b2631e5d379eb/transformers-4.49.0-py3-none-any.whl (10.0 MB)\n",
      "Collecting trl\n",
      "  Using cached https://mirrors.ustc.edu.cn/pypi/packages/dc/3c/51dab98b5d6636ba8de20551c81c2f76a503a9b33e969a0360e17ef79f9f/trl-0.15.0-py3-none-any.whl (318 kB)\n",
      "Collecting accelerate\n",
      "  Using cached https://mirrors.ustc.edu.cn/pypi/packages/0a/f6/791b9d7eb371a2f385da3b7f1769ced72ead7bf09744637ea2985c83d7ee/accelerate-1.4.0-py3-none-any.whl (342 kB)\n",
      "Collecting bitsandbytes\n",
      "  Using cached https://mirrors.ustc.edu.cn/pypi/packages/db/9d/9382259196d7ad7f3550702390081224e673a705e75b5660ee377b592fc0/bitsandbytes-0.45.2-py3-none-manylinux_2_24_x86_64.whl (69.7 MB)\n",
      "Requirement already satisfied: numpy>=1.17 in /app/anaconda/weitiao/lib/python3.12/site-packages (from peft) (2.1.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from peft) (24.2)\n",
      "Requirement already satisfied: psutil in /app/anaconda/weitiao/lib/python3.12/site-packages (from peft) (7.0.0)\n",
      "Requirement already satisfied: pyyaml in /app/anaconda/weitiao/lib/python3.12/site-packages (from peft) (6.0.2)\n",
      "Requirement already satisfied: torch>=1.13.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from peft) (2.6.0+cu126)\n",
      "Requirement already satisfied: tqdm in /app/anaconda/weitiao/lib/python3.12/site-packages (from peft) (4.67.1)\n",
      "Requirement already satisfied: safetensors in /app/anaconda/weitiao/lib/python3.12/site-packages (from peft) (0.5.2)\n",
      "Requirement already satisfied: huggingface-hub>=0.25.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from peft) (0.28.1)\n",
      "Requirement already satisfied: filelock in /app/anaconda/weitiao/lib/python3.12/site-packages (from transformers) (3.13.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /app/anaconda/weitiao/lib/python3.12/site-packages (from transformers) (2024.11.6)\n",
      "Requirement already satisfied: requests in /app/anaconda/weitiao/lib/python3.12/site-packages (from transformers) (2.32.3)\n",
      "Requirement already satisfied: tokenizers<0.22,>=0.21 in /app/anaconda/weitiao/lib/python3.12/site-packages (from transformers) (0.21.0)\n",
      "Requirement already satisfied: datasets>=2.21.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from trl) (3.3.0)\n",
      "Requirement already satisfied: rich in /app/anaconda/weitiao/lib/python3.12/site-packages (from trl) (13.9.4)\n",
      "Requirement already satisfied: pyarrow>=15.0.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from datasets>=2.21.0->trl) (19.0.0)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from datasets>=2.21.0->trl) (0.3.8)\n",
      "Requirement already satisfied: pandas in /app/anaconda/weitiao/lib/python3.12/site-packages (from datasets>=2.21.0->trl) (2.2.3)\n",
      "Requirement already satisfied: xxhash in /app/anaconda/weitiao/lib/python3.12/site-packages (from datasets>=2.21.0->trl) (3.5.0)\n",
      "Requirement already satisfied: multiprocess<0.70.17 in /app/anaconda/weitiao/lib/python3.12/site-packages (from datasets>=2.21.0->trl) (0.70.16)\n",
      "Requirement already satisfied: fsspec<=2024.12.0,>=2023.1.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from fsspec[http]<=2024.12.0,>=2023.1.0->datasets>=2.21.0->trl) (2024.6.1)\n",
      "Requirement already satisfied: aiohttp in /app/anaconda/weitiao/lib/python3.12/site-packages (from datasets>=2.21.0->trl) (3.11.12)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /app/anaconda/weitiao/lib/python3.12/site-packages (from huggingface-hub>=0.25.0->peft) (4.12.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /app/anaconda/weitiao/lib/python3.12/site-packages (from requests->transformers) (3.4.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /app/anaconda/weitiao/lib/python3.12/site-packages (from requests->transformers) (3.10)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /app/anaconda/weitiao/lib/python3.12/site-packages (from requests->transformers) (2.3.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /app/anaconda/weitiao/lib/python3.12/site-packages (from requests->transformers) (2025.1.31)\n",
      "Requirement already satisfied: setuptools in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (75.8.0)\n",
      "Requirement already satisfied: sympy==1.13.1 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (1.13.1)\n",
      "Requirement already satisfied: networkx in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (3.3)\n",
      "Requirement already satisfied: jinja2 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (3.1.4)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.6.77 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (12.6.77)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.6.77 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (12.6.77)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.6.80 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (12.6.80)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==9.5.1.17 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (9.5.1.17)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.6.4.1 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (12.6.4.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.3.0.4 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (11.3.0.4)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.7.77 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (10.3.7.77)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.7.1.2 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (11.7.1.2)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.5.4.2 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (12.5.4.2)\n",
      "Requirement already satisfied: nvidia-cusparselt-cu12==0.6.3 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (0.6.3)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.21.5 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (2.21.5)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.6.77 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (12.6.77)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12==12.6.85 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (12.6.85)\n",
      "Requirement already satisfied: triton==3.2.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from torch>=1.13.0->peft) (3.2.0)\n",
      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from sympy==1.13.1->torch>=1.13.0->peft) (1.3.0)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from rich->trl) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from rich->trl) (2.19.1)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from aiohttp->datasets>=2.21.0->trl) (2.4.6)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /app/anaconda/weitiao/lib/python3.12/site-packages (from aiohttp->datasets>=2.21.0->trl) (1.3.2)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from aiohttp->datasets>=2.21.0->trl) (25.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /app/anaconda/weitiao/lib/python3.12/site-packages (from aiohttp->datasets>=2.21.0->trl) (1.5.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /app/anaconda/weitiao/lib/python3.12/site-packages (from aiohttp->datasets>=2.21.0->trl) (6.1.0)\n",
      "Requirement already satisfied: propcache>=0.2.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from aiohttp->datasets>=2.21.0->trl) (0.2.1)\n",
      "Requirement already satisfied: yarl<2.0,>=1.17.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from aiohttp->datasets>=2.21.0->trl) (1.18.3)\n",
      "Requirement already satisfied: mdurl~=0.1 in /app/anaconda/weitiao/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich->trl) (0.1.2)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /app/anaconda/weitiao/lib/python3.12/site-packages (from jinja2->torch>=1.13.0->peft) (2.1.5)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /app/anaconda/weitiao/lib/python3.12/site-packages (from pandas->datasets>=2.21.0->trl) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /app/anaconda/weitiao/lib/python3.12/site-packages (from pandas->datasets>=2.21.0->trl) (2025.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /app/anaconda/weitiao/lib/python3.12/site-packages (from pandas->datasets>=2.21.0->trl) (2025.1)\n",
      "Requirement already satisfied: six>=1.5 in /app/anaconda/weitiao/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas->datasets>=2.21.0->trl) (1.17.0)\n",
      "Installing collected packages: transformers, bitsandbytes, accelerate, trl, peft\n",
      "Successfully installed accelerate-1.4.0 bitsandbytes-0.45.2 peft-0.14.0 transformers-4.49.0 trl-0.15.0\n"
     ]
    }
   ],
   "source": [
    "!pip uninstall peft transformers trl accelerate bitsandbytes -y\n",
    "!pip install peft transformers trl accelerate bitsandbytes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 49,
     "referenced_widgets": [
      "fc3993b236b24cbcaecc4e7192fe3254",
      "af30810fd0544ba4b904de2ce3f02cec",
      "1969437f6b7b4f92bd71058a0bd5755a",
      "83c6aa3318cf4c8c8be3fd4f8e9ba387",
      "a60b14d5d28547cea890f3c17ab2ac8a",
      "d02a1ccb5a914647a3e9464031d38b79",
      "5927fd687bc4479eae3bf5985445cc9c",
      "e5fd64b00bbb4e299930464604518bac",
      "3e6b9d78fce94a20bf09b22b12a58119",
      "4fcaab874f16414391964cd95ebaa648",
      "8c773d3f8c3842a887c37cf14ffcabee"
     ]
    },
    "editable": false,
    "id": "9AuTcgxB6b_z",
    "outputId": "4ed3992b-e42f-4400-edc9-51660f4b0eb8",
    "papermill": {
     "duration": 1.901541,
     "end_time": "2024-09-25T13:13:25.296367",
     "exception": false,
     "start_time": "2024-09-25T13:13:23.394826",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b74456a76f314ddd9780355b92f84c51",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Tokenizing train dataset (num_proc=2):   0%|          | 0/500 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70e28bba512f455ca52aafecbc473dda",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Tokenizing train dataset (num_proc=2):   0%|          | 0/500 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No label_names provided for model class `PeftModelForCausalLM`. Since `PeftModel` hides base models input arguments, if label_names is not given, label_names can't be set automatically within `Trainer`. Note that empty label_names list will be used instead.\n"
     ]
    }
   ],
   "source": [
    "from trl import SFTTrainer\n",
    "from transformers import TrainingArguments\n",
    "from unsloth import is_bfloat16_supported\n",
    "\n",
    "trainer = SFTTrainer(\n",
    "    model=model,\n",
    "    tokenizer=tokenizer,\n",
    "    train_dataset=dataset,\n",
    "    dataset_text_field=\"text\",\n",
    "    max_seq_length=max_seq_length,\n",
    "    dataset_num_proc=2,\n",
    "    args=TrainingArguments(\n",
    "        per_device_train_batch_size=2,\n",
    "        gradient_accumulation_steps=4,\n",
    "        # Use num_train_epochs = 1, warmup_ratio for full training runs!\n",
    "        warmup_steps=5,\n",
    "        max_steps=60,\n",
    "        learning_rate=2e-4,\n",
    "        fp16=not is_bfloat16_supported(),\n",
    "        bf16=is_bfloat16_supported(),\n",
    "        logging_steps=10,\n",
    "        optim=\"adamw_8bit\",\n",
    "        weight_decay=0.01,\n",
    "        lr_scheduler_type=\"linear\",\n",
    "        seed=3407,\n",
    "        output_dir=\"outputs\",\n",
    "    ),\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "editable": false,
    "id": "QosSqAta6b_z"
   },
   "source": [
    "## Model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 638
    },
    "editable": false,
    "id": "dGYprK9F6b_z",
    "outputId": "05b06c14-bfbb-4834-bf6f-80d66f91aff0",
    "papermill": {
     "duration": 1210.862521,
     "end_time": "2024-09-25T13:33:36.207919",
     "exception": false,
     "start_time": "2024-09-25T13:13:25.345398",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "==((====))==  Unsloth - 2x faster free finetuning | Num GPUs = 1\n",
      "   \\\\   /|    Num examples = 500 | Num Epochs = 1\n",
      "O^O/ \\_/ \\    Batch size per device = 2 | Gradient Accumulation steps = 4\n",
      "\\        /    Total batch size = 8 | Total steps = 60\n",
      " \"-____-\"     Number of trainable parameters = 41,943,040\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='60' max='60' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [60/60 03:25, Epoch 0/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>1.048500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>0.017100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>0.019400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>0.001900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>0.001600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60</td>\n",
       "      <td>0.001600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "trainer_stats = trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "editable": false,
    "id": "3-wyYeic6b_z"
   },
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<br>    <style><br>        .wandb-row {<br>            display: flex;<br>            flex-direction: row;<br>            flex-wrap: wrap;<br>            justify-content: flex-start;<br>            width: 100%;<br>        }<br>        .wandb-col {<br>            display: flex;<br>            flex-direction: column;<br>            flex-basis: 100%;<br>            flex: 1;<br>            padding: 10px;<br>        }<br>    </style><br><div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>train/epoch</td><td>▁▂▄▅▇██</td></tr><tr><td>train/global_step</td><td>▁▂▄▅▇██</td></tr><tr><td>train/grad_norm</td><td>█▅▂▂▁▁</td></tr><tr><td>train/learning_rate</td><td>█▇▅▄▂▁</td></tr><tr><td>train/loss</td><td>█▁▁▁▁▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>total_flos</td><td>1.792773518721024e+16</td></tr><tr><td>train/epoch</td><td>0.96</td></tr><tr><td>train/global_step</td><td>60</td></tr><tr><td>train/grad_norm</td><td>0.00141</td></tr><tr><td>train/learning_rate</td><td>0</td></tr><tr><td>train/loss</td><td>0.0016</td></tr><tr><td>train_loss</td><td>0.18168</td></tr><tr><td>train_runtime</td><td>210.2099</td></tr><tr><td>train_samples_per_second</td><td>2.283</td></tr><tr><td>train_steps_per_second</td><td>0.285</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run <strong style=\"color:#cdcd00\">silvery-donkey-18</strong> at: <a href='https://wandb.ai/neuqkyn-rhzz/my%20fint-tune%20on%20deepseek%20r1%20with%20medical%20data%202/runs/vrlu3wz2?apiKey=822b638c1f465585a2a139ce45f36c4d0a6e0180' target=\"_blank\">https://wandb.ai/neuqkyn-rhzz/my%20fint-tune%20on%20deepseek%20r1%20with%20medical%20data%202/runs/vrlu3wz2?apiKey=822b638c1f465585a2a139ce45f36c4d0a6e0180</a><br> View project at: <a href='https://wandb.ai/neuqkyn-rhzz/my%20fint-tune%20on%20deepseek%20r1%20with%20medical%20data%202?apiKey=822b638c1f465585a2a139ce45f36c4d0a6e0180' target=\"_blank\">https://wandb.ai/neuqkyn-rhzz/my%20fint-tune%20on%20deepseek%20r1%20with%20medical%20data%202?apiKey=822b638c1f465585a2a139ce45f36c4d0a6e0180</a><br>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20250219_145843-vrlu3wz2/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Save the fine-tuned model\n",
    "wandb.finish()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "editable": false,
    "id": "8yEZKVoo6b_z"
   },
   "source": [
    "## Model inference after fine-tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "editable": false,
    "id": "9kVwqsga6b_0",
    "outputId": "32d74201-9815-4093-960c-912550b5ad97",
    "papermill": {
     "duration": 7.941358,
     "end_time": "2024-09-25T13:33:44.195877",
     "exception": false,
     "start_time": "2024-09-25T13:33:36.254519",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$\n",
      "$$.\n",
      "$$\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$.\n",
      "$$\n"
     ]
    }
   ],
   "source": [
    "question = \"A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?\"\n",
    "\n",
    "\n",
    "FastLanguageModel.for_inference(model)  # Unsloth has 2x faster inference!\n",
    "inputs = tokenizer([prompt_style.format(question, \"\")], return_tensors=\"pt\").to(\"cuda\")\n",
    "\n",
    "outputs = model.generate(\n",
    "    input_ids=inputs.input_ids,\n",
    "    attention_mask=inputs.attention_mask,\n",
    "    max_new_tokens=1200,\n",
    "    use_cache=True,\n",
    ")\n",
    "response = tokenizer.batch_decode(outputs)\n",
    "print(response[0].split(\"### Response:\")[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "editable": false,
    "id": "TpM4uf7E6b_0",
    "outputId": "683efd00-33dc-46f6-833d-828c0fd275ea"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "<think>\n",
      "</think>\n",
      "</think>\n",
      "<think>\n",
      "</think>\n",
      "<think>\n",
      "<｜end▁of▁sentence｜>\n"
     ]
    }
   ],
   "source": [
    "question = \"A 59-year-old man presents with a fever, chills, night sweats, and generalized fatigue, and is found to have a 12 mm vegetation on the aortic valve. Blood cultures indicate gram-positive, catalase-negative, gamma-hemolytic cocci in chains that do not grow in a 6.5% NaCl medium. What is the most likely predisposing factor for this patient's condition?\"\n",
    "\n",
    "inputs = tokenizer([prompt_style.format(question, \"\")], return_tensors=\"pt\").to(\"cuda\")\n",
    "\n",
    "outputs = model.generate(\n",
    "    input_ids=inputs.input_ids,\n",
    "    attention_mask=inputs.attention_mask,\n",
    "    max_new_tokens=1200,\n",
    "    use_cache=True,\n",
    ")\n",
    "response = tokenizer.batch_decode(outputs)\n",
    "print(response[0].split(\"### Response:\")[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "editable": false,
    "id": "MbpJ1ZaY6b_0"
   },
   "source": [
    "## Saving the model locally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('DeepSeek-R1-Medical-COT-1/tokenizer_config.json',\n",
       " 'DeepSeek-R1-Medical-COT-1/special_tokens_map.json',\n",
       " 'DeepSeek-R1-Medical-COT-1/tokenizer.json')"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# new_model_online = \"kingabzpro/DeepSeek-R1-Medical-COT\"\n",
    "new_model_local = \"DeepSeek-R1-Medical-COT-1\"\n",
    "model.save_pretrained(new_model_local) # Local saving\n",
    "tokenizer.save_pretrained(new_model_local)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unsloth: Merging 4bit and LoRA weights to 16bit...\n",
      "Unsloth: Will use up to 357.38 out of 503.2 RAM for saving.\n",
      "Unsloth: Saving model... This might take 5 minutes ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 32/32 [00:00<00:00, 47.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unsloth: Saving tokenizer... Done.\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "new_model_local_merged = \"DeepSeek-R1-Medical-COT-1-merged\"\n",
    "model.save_pretrained_merged(new_model_local_merged, tokenizer, save_method = \"merged_16bit\",)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "editable": false,
    "id": "tju4gSOp6b_0"
   },
   "source": [
    "## Pushing the model to Hugging Face hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "editable": false,
    "id": "5I9egNBA6b_0",
    "papermill": {
     "duration": 14.531527,
     "end_time": "2024-09-25T13:34:04.065269",
     "exception": false,
     "start_time": "2024-09-25T13:33:49.533742",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'new_model_online' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[25], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m model\u001b[38;5;241m.\u001b[39mpush_to_hub(\u001b[43mnew_model_online\u001b[49m) \u001b[38;5;66;03m# Online saving\u001b[39;00m\n\u001b[1;32m      2\u001b[0m tokenizer\u001b[38;5;241m.\u001b[39mpush_to_hub(new_model_online) \u001b[38;5;66;03m# Online saving\u001b[39;00m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'new_model_online' is not defined"
     ]
    }
   ],
   "source": [
    "model.push_to_hub(new_model_online) # Online saving\n",
    "tokenizer.push_to_hub(new_model_online) # Online saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "editable": false,
    "id": "JkY85SIc6b_0"
   },
   "outputs": [],
   "source": [
    "\n",
    "model.push_to_hub_merged(new_model_online, tokenizer, save_method = \"merged_16bit\")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kaggle": {
   "accelerator": "nvidiaTeslaT4",
   "dataSources": [],
   "dockerImageVersionId": 30839,
   "isGpuEnabled": true,
   "isInternetEnabled": false,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1500.86091,
   "end_time": "2024-09-25T13:34:07.208125",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-09-25T13:09:06.347215",
   "version": "2.6.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "029a61b64a0844dab51a9eb8ab949bf0": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4e17d9034b6d41c58bdc8ca94ea0c2fd",
      "placeholder": "​",
      "style": "IPY_MODEL_ac02a01ce16e4fbca4ff1819a5153088",
      "value": " 52.9k/52.9k [00:00&lt;00:00, 4.40MB/s]"
     }
    },
    "045a773300914b91a43f7f4922baf8cc": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "08749a83ae4a47128e23b69dd791ab6e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "0a389bfe361f43578d55001f88a92fea": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "0f3a15d30fca4e5aa6321f1497e9c449": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1969437f6b7b4f92bd71058a0bd5755a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e5fd64b00bbb4e299930464604518bac",
      "max": 500,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_3e6b9d78fce94a20bf09b22b12a58119",
      "value": 500
     }
    },
    "1992bceec6b54ac2827285d39c8c12ee": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1cd217e8233d495db151f5debdf9f00b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_eb8becd866564ffd859e22bbc61a93e8",
      "max": 17209530,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_7a50ddf657a04aef8c68c4100323fcea",
      "value": 17209530
     }
    },
    "1f2e54775e6e4288a2a772b75d104fbb": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1fe6b27ff0cc4e9fa39b0b5840a25aec": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1992bceec6b54ac2827285d39c8c12ee",
      "placeholder": "​",
      "style": "IPY_MODEL_45f8a2d59bfe46008feebab8981d7146",
      "value": "model.safetensors: 100%"
     }
    },
    "2e82c8498b854d41842977ef8f9bbb66": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "30b0da59ccb241439ee988cf52bf81b3": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1f2e54775e6e4288a2a772b75d104fbb",
      "placeholder": "​",
      "style": "IPY_MODEL_e15cb3d0d72a475ea39eb1ef761dc827",
      "value": "tokenizer.json: 100%"
     }
    },
    "33edc00314bf444188376c01ea025038": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e68571618bf748ebbf6d3e244e0e7f29",
      "placeholder": "​",
      "style": "IPY_MODEL_e1d763ec083b49d0900cfe5d42d9b214",
      "value": "generation_config.json: 100%"
     }
    },
    "3914a21f18114d6784bbb8f4867be344": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3e6b9d78fce94a20bf09b22b12a58119": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "41c36bc4fcb542a0bbc3443cd4c742ea": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "424e62fec0c0416ba13f5863391e08f9": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "44d3cf803d244775826d566c4d9ca859": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "45f8a2d59bfe46008feebab8981d7146": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "4afe538bb09d4cf698235e135640ab6e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_1fe6b27ff0cc4e9fa39b0b5840a25aec",
       "IPY_MODEL_f811d4e961914901b3b458934512eb86",
       "IPY_MODEL_9812fb88a8b3411d8675d238161dd7aa"
      ],
      "layout": "IPY_MODEL_6dbb94366cc941a8ba336ba61ebf99ba"
     }
    },
    "4e17d9034b6d41c58bdc8ca94ea0c2fd": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4e3945399c1642c1bdf724f8a01fcd02": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_972543b62d0a4d908101b7abc8c1ca15",
      "max": 236,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_8e5e7c77e88a4d3b8b606f240b627a9d",
      "value": 236
     }
    },
    "4fcaab874f16414391964cd95ebaa648": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "51ad33f2e8dc44f79a8e8fd9a86e22b5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "5927fd687bc4479eae3bf5985445cc9c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "6dbb94366cc941a8ba336ba61ebf99ba": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "724109739173448fb1965e01895f51a2": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "76bc4ff5adb746a69954699cf5465c2f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_045a773300914b91a43f7f4922baf8cc",
      "placeholder": "​",
      "style": "IPY_MODEL_9ef2064efb05485b8b63bb3db815a8ef",
      "value": " 17.2M/17.2M [00:00&lt;00:00, 42.1MB/s]"
     }
    },
    "7a50ddf657a04aef8c68c4100323fcea": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "7ae90e66d3f1459ab5e0c202cf4228ab": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e5784f34cf184023ac60d15e4c347688",
      "placeholder": "​",
      "style": "IPY_MODEL_3914a21f18114d6784bbb8f4867be344",
      "value": "tokenizer_config.json: 100%"
     }
    },
    "80b57123cbb64b2c9a27c5159a104556": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "82860a16b8d944ff86f854f215535379": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_33edc00314bf444188376c01ea025038",
       "IPY_MODEL_4e3945399c1642c1bdf724f8a01fcd02",
       "IPY_MODEL_f58d5860eb854c698f6558e4422acced"
      ],
      "layout": "IPY_MODEL_0f3a15d30fca4e5aa6321f1497e9c449"
     }
    },
    "83c6aa3318cf4c8c8be3fd4f8e9ba387": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4fcaab874f16414391964cd95ebaa648",
      "placeholder": "​",
      "style": "IPY_MODEL_8c773d3f8c3842a887c37cf14ffcabee",
      "value": " 500/500 [00:02&lt;00:00, 99.67 examples/s]"
     }
    },
    "8c773d3f8c3842a887c37cf14ffcabee": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "8e5e7c77e88a4d3b8b606f240b627a9d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "972543b62d0a4d908101b7abc8c1ca15": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9812fb88a8b3411d8675d238161dd7aa": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_424e62fec0c0416ba13f5863391e08f9",
      "placeholder": "​",
      "style": "IPY_MODEL_0a389bfe361f43578d55001f88a92fea",
      "value": " 5.96G/5.96G [01:47&lt;00:00, 93.1MB/s]"
     }
    },
    "9ab8e04684054331928edd757ce8dea3": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9ef2064efb05485b8b63bb3db815a8ef": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a60b14d5d28547cea890f3c17ab2ac8a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ac02a01ce16e4fbca4ff1819a5153088": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "adb31ccfc7ef415fb47a40ede28fe9db": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "af30810fd0544ba4b904de2ce3f02cec": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d02a1ccb5a914647a3e9464031d38b79",
      "placeholder": "​",
      "style": "IPY_MODEL_5927fd687bc4479eae3bf5985445cc9c",
      "value": "Map (num_proc=2): 100%"
     }
    },
    "b01764e9ac74437aa174b132789417af": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_80b57123cbb64b2c9a27c5159a104556",
      "max": 52946,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_cf8df2dc586a4a9bba21066c50805fda",
      "value": 52946
     }
    },
    "b0dc37d789084b8e95ae90c155699ab7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_41c36bc4fcb542a0bbc3443cd4c742ea",
      "placeholder": "​",
      "style": "IPY_MODEL_08749a83ae4a47128e23b69dd791ab6e",
      "value": "special_tokens_map.json: 100%"
     }
    },
    "b8428a11dcfe4a2ea6d47b6e010fe3d2": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ba91815af2f84652987bd01420bb061f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bee5e75849cf448fa6c5eb2ef61f48d4": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cdc489ca63484d4abb3ac39a3f93e33c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_30b0da59ccb241439ee988cf52bf81b3",
       "IPY_MODEL_1cd217e8233d495db151f5debdf9f00b",
       "IPY_MODEL_76bc4ff5adb746a69954699cf5465c2f"
      ],
      "layout": "IPY_MODEL_bee5e75849cf448fa6c5eb2ef61f48d4"
     }
    },
    "cf82cef940944506bbb654363a48cdb7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ba91815af2f84652987bd01420bb061f",
      "max": 483,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_f500319ad888458f902f5fc4fc30d528",
      "value": 483
     }
    },
    "cf8df2dc586a4a9bba21066c50805fda": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "d02a1ccb5a914647a3e9464031d38b79": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "dca1f2da331542ae85818b544827dd96": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e07fdea2d68c4284b43df47b7f94fceb": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_b0dc37d789084b8e95ae90c155699ab7",
       "IPY_MODEL_cf82cef940944506bbb654363a48cdb7",
       "IPY_MODEL_fe6b626df24943978a0af261d29c7ee4"
      ],
      "layout": "IPY_MODEL_724109739173448fb1965e01895f51a2"
     }
    },
    "e15cb3d0d72a475ea39eb1ef761dc827": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e1d763ec083b49d0900cfe5d42d9b214": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e5784f34cf184023ac60d15e4c347688": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e5fd64b00bbb4e299930464604518bac": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e68571618bf748ebbf6d3e244e0e7f29": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "eb8becd866564ffd859e22bbc61a93e8": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f500319ad888458f902f5fc4fc30d528": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "f58d5860eb854c698f6558e4422acced": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b8428a11dcfe4a2ea6d47b6e010fe3d2",
      "placeholder": "​",
      "style": "IPY_MODEL_dca1f2da331542ae85818b544827dd96",
      "value": " 236/236 [00:00&lt;00:00, 17.9kB/s]"
     }
    },
    "f811d4e961914901b3b458934512eb86": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "danger",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2e82c8498b854d41842977ef8f9bbb66",
      "max": 5964186418,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_adb31ccfc7ef415fb47a40ede28fe9db",
      "value": 5964185850
     }
    },
    "fc3993b236b24cbcaecc4e7192fe3254": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_af30810fd0544ba4b904de2ce3f02cec",
       "IPY_MODEL_1969437f6b7b4f92bd71058a0bd5755a",
       "IPY_MODEL_83c6aa3318cf4c8c8be3fd4f8e9ba387"
      ],
      "layout": "IPY_MODEL_a60b14d5d28547cea890f3c17ab2ac8a"
     }
    },
    "fe6b626df24943978a0af261d29c7ee4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_44d3cf803d244775826d566c4d9ca859",
      "placeholder": "​",
      "style": "IPY_MODEL_51ad33f2e8dc44f79a8e8fd9a86e22b5",
      "value": " 483/483 [00:00&lt;00:00, 44.3kB/s]"
     }
    },
    "ff5beb41522f4b80bdb35c436f478bb6": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_7ae90e66d3f1459ab5e0c202cf4228ab",
       "IPY_MODEL_b01764e9ac74437aa174b132789417af",
       "IPY_MODEL_029a61b64a0844dab51a9eb8ab949bf0"
      ],
      "layout": "IPY_MODEL_9ab8e04684054331928edd757ce8dea3"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
